{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8D5tOVWvG6Ss",
    "outputId": "917998d3-b968-48d9-b7f1-a0f159ff1cd2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fcf180ad2f0>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import load_dataset\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "torch.manual_seed(12046)\n",
    "# Device: V100 16GB\n",
    "# If using a CPU, it will take a considerable amount of time. \n",
    "# Please consider reducing the model size to speed up the process (e.g., descrease n_layer)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GwjXDvhCG6Su",
    "outputId": "0f0cff6f-c785-4259-d271-91217c170986"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[ 1.0185, -1.3091,  1.2908,  0.5276],\n",
      "         [-0.2985,  1.6259,  2.0433, -0.6417],\n",
      "         [ 0.8795, -1.0512,  1.1491,  0.6116],\n",
      "         [ 0.2128, -0.5512,  0.0450,  0.5010]]])\n",
      "tensor([[[ 1.0185,    -inf,    -inf,    -inf],\n",
      "         [-0.2985,  1.6259,    -inf,    -inf],\n",
      "         [ 0.8795, -1.0512,  1.1491,    -inf],\n",
      "         [ 0.2128, -0.5512,  0.0450,  0.5010]]])\n",
      "tensor([[[1.0000, 0.0000, 0.0000, 0.0000],\n",
      "         [0.1274, 0.8726, 0.0000, 0.0000],\n",
      "         [0.4074, 0.0591, 0.5335, 0.0000],\n",
      "         [0.2743, 0.1278, 0.2319, 0.3659]]])\n"
     ]
    }
   ],
   "source": [
    "# Illustrate the role of mask in attention mechanism\n",
    "T = 4\n",
    "scores = torch.randn(1, T, T)\n",
    "print(scores)\n",
    "# Define lower triangular matrix\n",
    "tril = torch.tril(torch.ones(T, T))\n",
    "scores = scores.masked_fill(tril == 0, float('-inf'))\n",
    "print(scores)\n",
    "# Transform scores into weights between words\n",
    "print(F.softmax(scores, dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2o_FK0sUWW-t",
    "outputId": "af65728c-5920-4548-b8f5-5c9a925029b3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.0026) tensor(1.0010) tensor(4.0152)\n",
      "tensor(1.0026) tensor(1.0010) tensor(1.0038)\n",
      "tensor([[0.0921, 0.1476, 0.1698, 0.4256, 0.0489, 0.0599, 0.0172, 0.0388]])\n",
      "tensor([[0., 0., 0., 1., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "# The effect of increasing the variance of alignment scores.\n",
    "B, T, head_size = 32, 100, 16\n",
    "\n",
    "k = torch.randn(B, T, head_size)  # (B, T, H)\n",
    "q = torch.randn(B, T, head_size)  # (B, T, H)\n",
    "scores = q @ k.transpose(-2, -1)  # (B, T, T)\n",
    "print(k.std(), q.std(), scores.std())\n",
    "# The normalization of alignment scores.\n",
    "scores = scores / head_size ** 0.5\n",
    "print(k.std(), q.std(), scores.std())\n",
    "\n",
    "# The result of Softmax will be concentrated on one point, if the variance of data is too large.\n",
    "# This is why we need do normalization of alignment scores.\n",
    "x = torch.randn(1, 8)\n",
    "print(torch.softmax(x, dim=-1))\n",
    "print(torch.softmax(1000 * x, dim=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "ujPVSgIgG6Sv"
   },
   "outputs": [],
   "source": [
    "def attention(query, key, value, dropout, mask=None):\n",
    "    '''\n",
    "    Attention mechanism\n",
    "    \n",
    "    Args:\n",
    "    ----\n",
    "    query: torch.FloatTensor, Query in attention mechanism, shape (B, T, C)\n",
    "    key: torch.FloatTensor, Key in attention mechanism, shape (B, T, C)\n",
    "    value: torch.FloatTensor, Value in attention machanism, shape (B, T, C)\n",
    "    dropout: Dropout in attention machanism\n",
    "    mask: torch.FloatTensor，Mask in attention machanism, shape (T, T)\n",
    "    \n",
    "    Returns:\n",
    "    ----\n",
    "    out: torch.FloatTensor, Context vector, shape (B, T, C)\n",
    "    w_att: torch.FloatTensor, Weights, shape (B, T, T)\n",
    "    '''\n",
    "    # query, key, value have the same shape\n",
    "    B, T, C = query.shape\n",
    "    # (B, T, C) @ (B, C, T) --> (B, T, T)\n",
    "    scores = query @ key.transpose(-2, -1) / (C ** 0.5)\n",
    "    if mask is not None:\n",
    "        # If mask is None, it means the the word could use the context from both side.\n",
    "        # It is so-called bi-directional attention.\n",
    "        # If mask is lower triangular matrix, it is uni-directional attention.\n",
    "        scores = scores.masked_fill(mask == 0, float('-inf'))\n",
    "    w_att = dropout(F.softmax(scores, dim=-1))  # (B, T, T)\n",
    "    out = w_att @ value  # (B, T, C)\n",
    "    return out, w_att"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "SbJ2z5U3G6Sw"
   },
   "outputs": [],
   "source": [
    "class MaskedAttention(nn.Module):\n",
    "\n",
    "    def __init__(self, emb_size, head_size):\n",
    "        '''\n",
    "        Single head and uni-directional attention.\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        emb_size: int, The dimension of embedding.\n",
    "        head_size: int, The dimension of context.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.key = nn.Linear(emb_size, head_size, bias=False)\n",
    "        self.query = nn.Linear(emb_size, head_size, bias=False)\n",
    "        self.value = nn.Linear(emb_size, head_size, bias=False)\n",
    "        # This lower triangular matrix will NOT involve model training.\n",
    "        self.register_buffer(\n",
    "            'tril', torch.tril(torch.ones(sequence_len, sequence_len)))\n",
    "        self.dropout = nn.Dropout(0.4)\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        Forward pass\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        x: torch.FloatTensor, The tensor of text, shape (B, T, C). \n",
    "            B means batch size, T means text length, C means dimension of embedding (emb_size).\n",
    "        \n",
    "        Returns:\n",
    "        ----\n",
    "        out: torch.FloatTensor\n",
    "            The context according to attention mechanism, shape (B, T, H). H means dimension of context.\n",
    "        '''\n",
    "        B, T, C = x.shape\n",
    "        q = self.query(x)  # (B, T, H)\n",
    "        k = self.key(x)    # (B, T, H)\n",
    "        v = self.value(x)  # (B, T, H)\n",
    "        mask = self.tril[:T, :T]\n",
    "        out, _ = attention(q, k, v, self.dropout, mask)\n",
    "        return out         # (B, T, H)\n",
    "\n",
    "class MaskedMultiHeadAttention(nn.Module):\n",
    "\n",
    "    def __init__(self, emb_size, head_size):\n",
    "        '''\n",
    "        Multi-head and uni-directional attention.\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        emb_size: int, The dimension of embedding.\n",
    "        head_size: int, The dimension of context.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        assert(emb_size % head_size == 0)\n",
    "        # Define the number of head\n",
    "        n_head = emb_size // head_size\n",
    "        heads = [MaskedAttention(emb_size, head_size) for _ in range(n_head)]\n",
    "        self.heads = nn.ModuleList(heads)\n",
    "        # Linear projection\n",
    "        self.proj = nn.Linear(emb_size, emb_size)\n",
    "        self.dropout = nn.Dropout(0.4)\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        Forward pass\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        x: torch.FloatTensor, The tensor of text, shape (B, T, C). \n",
    "            B means batch size, T means text length, C means dimension of embedding (emb_size).\n",
    "        \n",
    "        Returns:\n",
    "        ----\n",
    "        out: torch.FloatTensor\n",
    "            The context according to attention mechanism, shape (B, T, C).\n",
    "        '''\n",
    "        # Concatenate the results of single head attention\n",
    "        out = torch.cat([h(x) for h in self.heads], dim=-1) # (B, T, C)\n",
    "        out = self.dropout(self.proj(out))\n",
    "        return out\n",
    "\n",
    "class FeedForward(nn.Module):\n",
    "\n",
    "    def __init__(self, emb_size):\n",
    "        '''\n",
    "        Multilayer perceptron.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.l1 = nn.Linear(emb_size, 4 * emb_size)\n",
    "        self.l2 = nn.Linear(4 * emb_size, emb_size)\n",
    "        self.dropout = nn.Dropout(0.4)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.gelu(self.l1(x))\n",
    "        out = self.dropout(self.l2(x))\n",
    "        return out\n",
    "\n",
    "class Block(nn.Module):\n",
    "\n",
    "    def __init__(self, emb_size, head_size):\n",
    "        '''\n",
    "        Decode block\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        emb_size: int, The dimension of embedding.\n",
    "        head_size: int, The head size of single head attention.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.mha = MaskedMultiHeadAttention(emb_size, head_size)\n",
    "        self.ff = FeedForward(emb_size)\n",
    "        # Layer normalization\n",
    "        self.ln1 = nn.LayerNorm(emb_size)\n",
    "        self.ln2 = nn.LayerNorm(emb_size)\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        Forward pass\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        x: torch.FloatTensor, The tensor of text, shape (B, T, C). \n",
    "        \n",
    "        Returns:\n",
    "        ----\n",
    "        out: torch.FloatTensor, The output of decode block, shape (B, T, C).\n",
    "        '''\n",
    "        # Residual connections\n",
    "        x = x + self.mha(self.ln1(x))   # (B, T, C)\n",
    "        out = x + self.ff(self.ln2(x))  # (B, T, C)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "KlDmjSCMG6Sx"
   },
   "outputs": [],
   "source": [
    "# Some parameters\n",
    "emb_size = 128\n",
    "head_size = 8\n",
    "n_layer = 12\n",
    "sequence_len = 64\n",
    "learning_rate = 1e-3\n",
    "eval_iters = 20\n",
    "batch_size=500\n",
    "# If GPU exists, use GPU to do computation.\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "fwr_fSVpG6Sx"
   },
   "outputs": [],
   "source": [
    "class CharGPT(nn.Module):\n",
    "\n",
    "    def __init__(self, vs):\n",
    "        '''\n",
    "        Use GPT-2 to learn python script.\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        vs: int, The size of dictionary.\n",
    "        '''\n",
    "        super().__init__()\n",
    "        # The embedding of token\n",
    "        self.token_embedding = nn.Embedding(vs, emb_size)\n",
    "        # The embedding of position\n",
    "        self.position_embedding = nn.Embedding(sequence_len, emb_size)\n",
    "        # Decode blocks\n",
    "        blocks = [Block(emb_size, head_size) for _ in range(n_layer)]\n",
    "        self.blocks = nn.Sequential(*blocks)\n",
    "        self.ln = nn.LayerNorm(emb_size)\n",
    "        # Language modeling head\n",
    "        self.lm_head = nn.Linear(emb_size, vs)\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        Forward pass\n",
    "        \n",
    "        Args:\n",
    "        ----\n",
    "        x: torch.LongTensor, The dictionary embedding of text, shape (B, T). \n",
    "        \n",
    "        Returns:\n",
    "        ----\n",
    "        logits: torch.FloatTensor, The prediction, shape (B, T, vs).\n",
    "        '''\n",
    "        B, T = x.shape\n",
    "        # Define the position of token, shape (T)\n",
    "        pos = torch.arange(0, T, dtype=torch.long, device=x.device)\n",
    "        tok_emb = self.token_embedding(x)       # (B, T,  C)\n",
    "        pos_emb = self.position_embedding(pos)  # (   T,  C)\n",
    "        x = tok_emb + pos_emb                   # (B, T,  C)\n",
    "        x = self.blocks(x)                      # (B, T,  C)\n",
    "        x = self.ln(x)                          # (B, T,  C)\n",
    "        logits = self.lm_head(x)                # (B, T, vs)\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8LkvZyGjG6Sx",
    "outputId": "05aa4349-4ba4-4516-ee94-a19a08fe65cf"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets = load_dataset('code_search_net', 'python')\n",
    "datasets = raw_datasets['train'].filter(lambda x: 'apache/spark' in x['repository_name'])\n",
    "\n",
    "class char_tokenizer:\n",
    "\n",
    "    def __init__(self, data):\n",
    "        # The dictionary contains all chars in data.\n",
    "        chars = sorted(list(set(''.join(data))))\n",
    "        # <|e|> means the end of text.\n",
    "        self.char2ind = {s : i + 1 for i, s in enumerate(chars)}\n",
    "        self.char2ind['<|e|>'] = 0\n",
    "        self.ind2char = {i : s for s, i in self.char2ind.items()}\n",
    "\n",
    "    def encode(self, text):\n",
    "        return [self.char2ind[c] for c in text]\n",
    "\n",
    "    def decode(self, enc):\n",
    "        if isinstance(enc, int):\n",
    "            return self.ind2char[enc]\n",
    "        return [self.ind2char[i] for i in enc]\n",
    "\n",
    "tok = char_tokenizer(datasets['whole_func_string'])\n",
    "len(tok.char2ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "No_AQspwG6Sx",
    "outputId": "132ff3b8-1399-4982-c7a5-9400eab93eca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2408290 parameters\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "CharGPT(\n",
       "  (token_embedding): Embedding(98, 128)\n",
       "  (position_embedding): Embedding(64, 128)\n",
       "  (blocks): Sequential(\n",
       "    (0): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (1): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (2): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (3): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (4): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (5): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (6): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (7): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (8): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (9): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (10): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "    (11): Block(\n",
       "      (mha): MaskedMultiHeadAttention(\n",
       "        (heads): ModuleList(\n",
       "          (0-15): 16 x MaskedAttention(\n",
       "            (key): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (query): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (value): Linear(in_features=128, out_features=8, bias=False)\n",
       "            (dropout): Dropout(p=0.4, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (proj): Linear(in_features=128, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ff): FeedForward(\n",
       "        (l1): Linear(in_features=128, out_features=512, bias=True)\n",
       "        (l2): Linear(in_features=512, out_features=128, bias=True)\n",
       "        (dropout): Dropout(p=0.4, inplace=False)\n",
       "      )\n",
       "      (ln1): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "      (ln2): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "    )\n",
       "  )\n",
       "  (ln): LayerNorm((128,), eps=1e-05, elementwise_affine=True)\n",
       "  (lm_head): Linear(in_features=128, out_features=98, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the structure of model\n",
    "model = CharGPT(len(tok.char2ind)).to(device)\n",
    "# Print the number of model parameters\n",
    "print(f'{sum(p.numel() for p in model.parameters())} parameters')\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "VOl4s229G6Sy"
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def generate_batch(model, idx, max_new_tokens=300):\n",
    "    '''\n",
    "    Use model to generate text (recurrently use model to do inference).\n",
    "    \n",
    "    Args:\n",
    "    ----\n",
    "    model: CharGPT\n",
    "    idx: torch.LongTensor, The position in the dictionary, shape (1, T)\n",
    "    max_new_tokens: int, The max length of new tokens.\n",
    "    \n",
    "    Returns:\n",
    "    ----\n",
    "    out: list[int], The generated text.\n",
    "    '''\n",
    "    # Put the model on evaluation mode\n",
    "    model.eval()\n",
    "    for _ in range(max_new_tokens):\n",
    "        # Limit the context length\n",
    "        context = idx[:, -sequence_len:]\n",
    "        # The efficiency is quite low, as there are many useless calculation\n",
    "        logits = model(context)\n",
    "        # Use the last logits\n",
    "        logits = logits[:, -1, :]\n",
    "        probs = F.softmax(logits, dim=-1)\n",
    "        # Generate the prediction according to the probs\n",
    "        # This step is stochastic\n",
    "        ix = torch.multinomial(probs, num_samples=1)\n",
    "        idx = torch.cat((idx, ix), dim=1)\n",
    "        if ix.item() == 0:\n",
    "            break\n",
    "    # Put the model on train mode\n",
    "    model.train()\n",
    "    return idx.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HQizE-2mG6Sz",
    "outputId": "21033dbb-9a8c-4ea2-9692-fcc000227b4c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def* O(h/of(\"YP`soE f|dwöR:1'_v?Q9)Nsx/Q=CKf\\M:iKcaI%+Q3m<sAegWSJ?0Nx!qJT3\\yMm5Za)'W*N\\WmsB?({<|e|>\n"
     ]
    }
   ],
   "source": [
    "# Test text generation\n",
    "begin_text = torch.tensor(tok.encode('def'), device=device).unsqueeze(0)\n",
    "print(''.join(tok.decode(generate_batch(model, begin_text))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "q0Fw0TCyG6Sz",
    "outputId": "140dc25b-792d-45fa-a8c3-b975461237a7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([605913, 64]), torch.Size([605913, 64]))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process(data, sequence_len=sequence_len):\n",
    "    '''\n",
    "    Transform text to train data.\n",
    "    '''\n",
    "    # text is list[str]\n",
    "    text = data['whole_func_string']\n",
    "    inputs, labels = [], []\n",
    "    for i in text:\n",
    "        enc = tok.encode(i)\n",
    "        # 0 means the end of text\n",
    "        enc += [0]\n",
    "        # Transform one text to multiple train data\n",
    "        for i in range(len(enc) - sequence_len):\n",
    "            inputs.append(enc[i: i + sequence_len])\n",
    "            # The label is the next char. It is sufficient to move one position.\n",
    "            labels.append(enc[i + 1: i + 1 + sequence_len])\n",
    "    return {'inputs': inputs, 'labels': labels}\n",
    "\n",
    "# Split data into train set and test set\n",
    "tokenized = datasets.train_test_split(test_size=0.1, seed=1024, shuffle=True)\n",
    "# The train data merely contains inputs and labels.\n",
    "tokenized = tokenized.map(process, batched=True, remove_columns=datasets.column_names)\n",
    "tokenized.set_format(type='torch', device=device)\n",
    "\n",
    "tokenized['train']['inputs'].shape, tokenized['train']['labels'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-x4i2q1iG6S0",
    "outputId": "23ec73d3-566d-4bfa-98c0-d3c05be82e70"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'inputs': tensor([[ 2,  2,  2,  ...,  2,  2,  4],\n",
       "         [81, 80, 88,  ...,  2,  2, 10],\n",
       "         [ 4, 37, 84,  ...,  2,  2,  2],\n",
       "         ...,\n",
       "         [75, 85,  2,  ..., 70, 71, 84],\n",
       "         [ 2,  2,  2,  ..., 67, 78, 53],\n",
       "         [87, 84, 67,  ..., 89,  2, 38]], device='cuda:0'),\n",
       " 'labels': tensor([[ 2,  2, 32,  ...,  2,  4,  4],\n",
       "         [80, 88, 71,  ...,  2, 10, 70],\n",
       "         [37, 84, 71,  ...,  2,  2,  2],\n",
       "         ...,\n",
       "         [85,  2, 72,  ..., 71, 84, 75],\n",
       "         [ 2,  2,  2,  ..., 78, 53, 81],\n",
       "         [84, 67, 86,  ...,  2, 38, 53]], device='cuda:0')}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build data loader\n",
    "train_loader = DataLoader(tokenized['train'], batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(tokenized['test'], batch_size=batch_size, shuffle=True)\n",
    "# Get one batch data\n",
    "next(iter(test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QD37LTDbG6S0",
    "outputId": "b51ce4ce-7f4a-41dc-c6a2-f518afaa9ce1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 4.730088233947754, 'test': 4.726046085357666}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def estimate_loss(model):\n",
    "    re = {}\n",
    "    # Put the model on evaluation mode\n",
    "    model.eval()\n",
    "    re['train'] = _loss(model, train_loader)\n",
    "    re['test'] = _loss(model, test_loader)\n",
    "    # Put the model on train mode\n",
    "    model.train()\n",
    "    return re\n",
    "\n",
    "@torch.no_grad()\n",
    "def _loss(model, data_loader):\n",
    "    '''\n",
    "    Measure the performance of model based on different data sets.\n",
    "    '''\n",
    "    loss = []\n",
    "    data_iter= iter(data_loader)\n",
    "    # Use eval_iters batch data to measure the performance\n",
    "    for k in range(eval_iters):\n",
    "        data = next(data_iter, None)\n",
    "        if data is None:\n",
    "            data_iter = iter(data_loader)\n",
    "            data = next(data_iter, None)\n",
    "        inputs, labels = data['inputs'], data['labels']\n",
    "        logits = model(inputs)\n",
    "        # According to the definition of cross_entropy in PyTorch,\n",
    "        # we need to transpose the logits.\n",
    "        # More details can be found in official document.\n",
    "        logits = logits.transpose(-2, -1)\n",
    "        loss.append(F.cross_entropy(logits, labels).item())\n",
    "    return torch.tensor(loss).mean().item()\n",
    "\n",
    "estimate_loss(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "TgKhC5TmG6S0"
   },
   "outputs": [],
   "source": [
    "def train_gpt(model, optimizer, data_loader, epochs=10):\n",
    "    lossi = []\n",
    "    for epoch in range(epochs):\n",
    "        for i, data in enumerate(data_loader, 0):\n",
    "            inputs, labels = data['inputs'], data['labels']\n",
    "            optimizer.zero_grad()\n",
    "            logits = model(inputs)\n",
    "            # According to the definition of cross_entropy in PyTorch,\n",
    "            # we need to transpose the logits.\n",
    "            # More details can be found in official document.\n",
    "            logits = logits.transpose(-2, -1)\n",
    "            loss = F.cross_entropy(logits, labels)\n",
    "            lossi.append(loss.item())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        # Evaluate the performance\n",
    "        stats = estimate_loss(model)\n",
    "        train_loss = f'train loss {stats['train']:.4f}'\n",
    "        test_loss = f'test loss {stats['test']:.4f}'\n",
    "        print(f'epoch {epoch:>2}: {train_loss}, {test_loss}')\n",
    "    return lossi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MCPIFH2dG6S1",
    "outputId": "68de3a92-b717-4147-ea07-bf456bae2190"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 0.9037, test loss 1.1066\n",
      "epoch  1: train loss 0.7246, test loss 1.0086\n",
      "epoch  2: train loss 0.6448, test loss 0.9719\n",
      "epoch  3: train loss 0.5838, test loss 0.9607\n",
      "epoch  4: train loss 0.5468, test loss 0.9672\n",
      "epoch  5: train loss 0.5156, test loss 0.9663\n",
      "epoch  6: train loss 0.4891, test loss 0.9596\n",
      "epoch  7: train loss 0.4687, test loss 0.9652\n",
      "epoch  8: train loss 0.4517, test loss 0.9709\n",
      "epoch  9: train loss 0.4347, test loss 0.9761\n"
     ]
    }
   ],
   "source": [
    "l = train_gpt(model, optim.AdamW(model.parameters(), lr=learning_rate), train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 448
    },
    "id": "pgRJxHwOG6S1",
    "outputId": "8787891f-92aa-4eca-a09b-83b8e5fdd3c9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fcdb82aabc0>]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5R0lEQVR4nO3deXxU9b3/8fcsyUxCMpONZBKSsAiC7AgiAa9bI4hcK22vtdZbcO2lF+6V2kXRavvz/my819veLtfrclvlZy1irYItUhRBoJbIHiSgLLIkhEzCksxk3+b8/ggZCJCQQDInybyej8c8ZM4y8znfBzLvx/d8v99jMQzDEAAAgEmsZhcAAADCG2EEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqu9kFdEQgENCxY8cUGxsri8VidjkAAKADDMNQRUWF0tLSZLW23f/RK8LIsWPHlJGRYXYZAADgEhQWFio9Pb3N/b0ijMTGxkpqvhiXy2VyNQAAoCP8fr8yMjKCv+Nt6RVhpOXWjMvlIowAANDLXGyIBQNYAQCAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqwggAADBVr3hQXnf5zV8P6mhZjb4xOUMjPDyADwAAM4R1z8h7u4q1eONhFZysNrsUAADCVliHEevpRxoHDJMLAQAgjIV1GLGc/q9hkEYAADBLWIeRlp4RoggAAOYJ6zDS0jUSoGcEAADThHUYsZ4OI2QRAADME9ZhxKKWAaykEQAAzBLWYcQa1lcPAEDPENY/x2em9tIzAgCAWcI6jLQIBMyuAACA8BXWYYSpvQAAmC+sw4iFqb0AAJgurMNIS88IXSMAAJgnzMNI83/pGQEAwDxhHUZalmAligAAYJ6wDiP0jAAAYL6wDiNnBrCaWwcAAOEsrMPImQGspBEAAMxCGBE9IwAAmCmsw4iCHSOkEQAAzBLWYYSeEQAAzBfWYeR0xwhTewEAMFFYhxErt2kAADBdWIcRS/A2DWEEAACzhHkYaf4vWQQAAPOEdRhhACsAAOYL6zByZgAraQQAALOEdRhp6RnhNg0AAOYJ6zBiYTYNAACmC/MwwpgRAADMFtZhxBp8ai9pBAAAs4R1GGFqLwAA5gvrMHJmACtpBAAAs4R1GOHZNAAAmC+8wwjLwQMAYLqwDiOsMwIAgPnCOoxYgrNpzK0DAIBwFtZhxMqiZwAAmC6sw0jLmBGiCAAA5gnzMNL83wD3aQAAME2nwsgLL7ygsWPHyuVyyeVyKSsrS3/5y1/aPeett97SiBEj5HQ6NWbMGK1cufKyCu5KFtEzAgCA2ToVRtLT0/Xss89q27Zt2rp1q26++Wbdcccd2r179wWP37hxo+6++2498MAD2rFjh2bPnq3Zs2crPz+/S4q/XCwHDwCA+SzGZY7eTEhI0HPPPacHHnjgvH133XWXqqqqtGLFiuC2KVOmaPz48XrxxRc7/B1+v19ut1s+n08ul+tyym3lP9/fq//+6IDunTpIP/nyqC77XAAA0PHf70seM9LU1KSlS5eqqqpKWVlZFzwmNzdX2dnZrbbNmDFDubm57X52XV2d/H5/q1d3sDCbBgAA03U6jOzatUsxMTFyOByaN2+eli1bppEjR17wWK/Xq5SUlFbbUlJS5PV62/2OnJwcud3u4CsjI6OzZXbImRVYu+XjAQBAB3Q6jAwfPlx5eXnatGmTvvOd72ju3Lnas2dPlxa1aNEi+Xy+4KuwsLBLP7/FmWfTkEYAADCLvbMnREZGaujQoZKkiRMnasuWLfrlL3+pl1566bxjPR6PSkpKWm0rKSmRx+Np9zscDoccDkdnS+s0Kz0jAACY7rLXGQkEAqqrq7vgvqysLK1Zs6bVttWrV7c5xiTUzqzAam4dAACEs071jCxatEgzZ85UZmamKioqtGTJEq1bt07vv/++JGnOnDkaMGCAcnJyJEkPP/ywbrjhBv3sZz/TrFmztHTpUm3dulUvv/xy11/JJWAAKwAA5utUGCktLdWcOXNUXFwst9utsWPH6v3339ctt9wiSSooKJDVeqazZerUqVqyZIl+9KMf6fHHH9ewYcO0fPlyjR49umuv4hJZeGovAACm61QY+e1vf9vu/nXr1p237c4779Sdd97ZqaJCxcKiZwAAmC6sn03DAFYAAMwX5mGk+b9M7QUAwDxhHUaCD8ojiwAAYJrwDiPMpgEAwHRhHkYYMwIAgNnCOoycGTMCAADMEtZhpOXZNEztBQDAPGEdRqzWlgGshBEAAMwS1mGEFVgBADBfeIeR0/9tYgQrAACmCeswYre2zKYhjAAAYJbwDiO25stvpGcEAADThHcYOd0z0thEGAEAwCzhHUZsp8NIIGByJQAAhK/wDiOne0YYwAoAgHnCOozYrM2X38BtGgAATBPWYaTlNg09IwAAmCe8w8jp2zQNTYwZAQDALGEdRmyMGQEAwHRhHUYiTq8zQhgBAMA8YR1GWnpGGpjaCwCAacI6jASn9jKbBgAA04R5GGE5eAAAzBbeYSS4AithBAAAs4R3GAk+m4YxIwAAmCXMwwizaQAAMFtYhxGbrWU2DWEEAACzhHUYiWDRMwAATBfWYeTsFVgNg0ACAIAZwjqMtIwZkZhRAwCAWcI7jJweMyJxqwYAALOEdRhpuU0j8eReAADMEtZhJNJ25vLrGwkjAACYIazDiNVqCQaSOsIIAACmCOswIkmR9uYmoGcEAABzhH0YcdjpGQEAwExhH0boGQEAwFxhH0bO9Iw0mVwJAADhKezDCD0jAACYizDS0jPCOiMAAJgi7MOIw26TJNU1EEYAADBD2IeRlnVG6ukZAQDAFGEfRhwRp2/TNDCAFQAAM4R9GKFnBAAAc4V9GHFEMGYEAAAzhX0YoWcEAABzhX0YOTNmhDACAIAZwj6MnOkZYQArAABmCPswElwOnp4RAABMQRixM2YEAAAzdSqM5OTk6JprrlFsbKySk5M1e/Zs7d27t91zFi9eLIvF0urldDovq+iuxLNpAAAwV6fCyPr16zV//nx98sknWr16tRoaGjR9+nRVVVW1e57L5VJxcXHwdeTIkcsquisFl4MnjAAAYAp7Zw5etWpVq/eLFy9WcnKytm3bpuuvv77N8ywWizwez6VV2M3oGQEAwFyXNWbE5/NJkhISEto9rrKyUgMHDlRGRobuuOMO7d69u93j6+rq5Pf7W726S3AAayOzaQAAMMMlh5FAIKCFCxdq2rRpGj16dJvHDR8+XK+88oreffddvf766woEApo6daqOHj3a5jk5OTlyu93BV0ZGxqWWeVGRwTBCzwgAAGa45DAyf/585efna+nSpe0el5WVpTlz5mj8+PG64YYb9M4776h///566aWX2jxn0aJF8vl8wVdhYeGllnlRjBkBAMBcnRoz0mLBggVasWKFNmzYoPT09E6dGxERoQkTJujAgQNtHuNwOORwOC6ltE6jZwQAAHN1qmfEMAwtWLBAy5Yt09q1azV48OBOf2FTU5N27dql1NTUTp/bHaKCD8pjzAgAAGboVM/I/PnztWTJEr377ruKjY2V1+uVJLndbkVFRUmS5syZowEDBignJ0eS9PTTT2vKlCkaOnSoysvL9dxzz+nIkSN68MEHu/hSLk20ozmMVNY1mlwJAADhqVNh5IUXXpAk3Xjjja22v/rqq7r33nslSQUFBbJaz3S4lJWV6aGHHpLX61V8fLwmTpyojRs3auTIkZdXeReJcTQ3QXU9PSMAAJjBYhiGYXYRF+P3++V2u+Xz+eRyubr0s4+WVeu6f/9IDrtVe//vzC79bAAAwllHf7/D/tk0/SKbe0bqGgNq5Pk0AACEHGHEceZOVRW3agAACLmwDyORdqsibBZJUnU9g1gBAAi1sA8j0pnekSpm1AAAEHKEEUkuZ4QkyVdDGAEAINQII5LiolvCSL3JlQAAEH4II5LcUc1hpLy6weRKAAAIP4QRSXHRkZIIIwAAmIEwIimupWekhjACAECoEUZ01piRasaMAAAQaoQRnTVmhJ4RAABCjjAixowAAGAmwogYMwIAgJkII2LMCAAAZiKMiDEjAACYiTAiyR1cgbVBgYBhcjUAAIQXwojO9IwYhlRRy/NpAAAIJcKIJIfdpuhImySpnOfTAAAQUoSR0+J4Pg0AAKYgjJyWENO81kjuwZMmVwIAQHghjJz2pREpkqRPCCMAAIQUYeS0qwfGS5KKy2tNrgQAgPBCGDltQJxTknTMV2NyJQAAhBfCyGmp7ihJzVN7K2oZxAoAQKgQRk7r57DL5bRLkop93KoBACBUCCNnSYtr7h05Vs6tGgAAQoUwcpaWMELPCAAAoUMYOUuKq3kQa4mfMAIAQKgQRs6S4nJIIowAABBKhJGzeII9I3UmVwIAQPggjJyF2zQAAIQeYeQsydymAQAg5AgjZ2npGTlRWa+GpoDJ1QAAEB4II2dJiI5UhM0iSTpewbgRAABCgTByFqvVouTY5t4RL7dqAAAICcLIOVrGjZQSRgAACAnCyDlSYpneCwBAKBFGzsHCZwAAhBZh5BwJ/ZrDyKmqepMrAQAgPBBGzpEQEymJMAIAQKgQRs6REN0cRsqqCSMAAIQCYeQcCf2aw8hJekYAAAgJwsg5Ek/fpikjjAAAEBKEkXPEn75NU17ToKaAYXI1AAD0fYSRc8RHR0iSDEMqZ9wIAADdjjByDrvNKndUcyBhECsAAN2PMHIBiS2DWCsJIwAAdDfCyAW0DGI9QRgBAKDbEUYuICmmeRXWE5U8nwYAgO5GGLmAljByvIIwAgBAd+tUGMnJydE111yj2NhYJScna/bs2dq7d+9Fz3vrrbc0YsQIOZ1OjRkzRitXrrzkgkOhfyw9IwAAhEqnwsj69es1f/58ffLJJ1q9erUaGho0ffp0VVVVtXnOxo0bdffdd+uBBx7Qjh07NHv2bM2ePVv5+fmXXXx3ie/HkvAAAISKxTCMS17Z6/jx40pOTtb69et1/fXXX/CYu+66S1VVVVqxYkVw25QpUzR+/Hi9+OKLHfoev98vt9stn88nl8t1qeV22J92HtO/vrFDU4YkaOm3s7r9+wAA6Is6+vt9WWNGfD6fJCkhIaHNY3Jzc5Wdnd1q24wZM5Sbm9vmOXV1dfL7/a1eodSyzoivpjGk3wsAQDi65DASCAS0cOFCTZs2TaNHj27zOK/Xq5SUlFbbUlJS5PV62zwnJydHbrc7+MrIyLjUMi9JSxjx1zSE9HsBAAhHlxxG5s+fr/z8fC1durQr65EkLVq0SD6fL/gqLCzs8u9oz5meEcIIAADdzX4pJy1YsEArVqzQhg0blJ6e3u6xHo9HJSUlrbaVlJTI4/G0eY7D4ZDD4biU0rpESxiprGtUY1NAdhszoAEA6C6d+pU1DEMLFizQsmXLtHbtWg0ePPii52RlZWnNmjWttq1evVpZWT13YKjLeSaj+WsZNwIAQHfqVBiZP3++Xn/9dS1ZskSxsbHyer3yer2qqakJHjNnzhwtWrQo+P7hhx/WqlWr9LOf/Uyff/65fvKTn2jr1q1asGBB111FF7PbrIpxNAcSbtUAANC9OhVGXnjhBfl8Pt14441KTU0Nvt58883gMQUFBSouLg6+nzp1qpYsWaKXX35Z48aN0x//+EctX7683UGvPQHjRgAACI1OjRnpyJIk69atO2/bnXfeqTvvvLMzX2U6V1SEisprCCMAAHQzRma2wR3FbRoAAEKBMNKGhNNLwp/gYXkAAHQrwkgbPK4oSVKJv9bkSgAA6NsII21Ii3NKkop9hBEAALoTYaQNaXHNPSN7iv0dGrgLAAAuDWGkDdOGJslutehAaaWOltVc/AQAAHBJCCNtcEdFyONuvlVzvJJBrAAAdBfCSDsSY5qfj3Oyst7kSgAA6LsII+1IOj299yQ9IwAAdBvCSDsSY06HkSp6RgAA6C6EkXZwmwYAgO5HGGlHYsttmipu0wAA0F0II+0I3qahZwQAgG5DGGlHYr/m2zQnGMAKAEC3IYy0o39scxg5zsPyAADoNoSRdqSeXvTsZFW9ahuaTK4GAIC+iTDSDndUhKIibJIkLw/MAwCgWxBG2mGxWJR6+um9x3w8nwYAgO5AGLmINHfz03uLy+kZAQCgOxBGLqJl3EgxPSMAAHQLwshFpMY194wcY8wIAADdgjByEWktPSPl9IwAANAdCCMX0dIzUkzPCAAA3YIwchEtPSPH6BkBAKBbEEYuoqVnxF/bqKq6RpOrAQCg7yGMXESMw65Yp10SM2oAAOgOhJEOaFlr5BhrjQAA0OUIIx3QsgorPSMAAHQ9wkgHpNIzAgBAtyGMdEAaq7ACANBtCCMd0DKjpvAUYQQAgK5GGOmAEZ5YSdLuYz4ZhmFyNQAA9C2EkQ4Y7omVM8Iqf22jtheUmV0OAAB9CmGkAyJsVt06yiNJWvNZqcnVAADQtxBGOmh8RpwkaV9JpbmFAADQxxBGOujKlOZxIwdKK0yuBACAvoUw0kHp8dGSmp/eyyBWAAC6DmGkg5JdDklSXWNA5dUNJlcDAEDfQRjpIGeETYn9IiVJx1j8DACALkMY6YRhKTGSpO0F5eYWAgBAH0IY6YSsIUmSpB2sNQIAQJchjHTCkP79JEmFp6pNrgQAgL6DMNIJmQnNM2qOnCSMAADQVQgjnTC4fz9ZLFJpRZ1KK2rNLgcAgD6BMNIJLmeERnhckqQ/5R0zuRoAAPoGwkgnfe3qAZKkD/aUmFwJAAB9A2GkkyZkxkmSispYawQAgK5AGOmktLgoSZLXX6umAMvCAwBwuQgjnZQc65TdalFTwFCJn0GsAABcLsJIJ9msFnncTknSsXJu1QAAcLkII5dgwOlbNUWEEQAALlunw8iGDRt0++23Ky0tTRaLRcuXL2/3+HXr1slisZz38nq9l1qz6dLjmxc/++J4lcmVAADQ+3U6jFRVVWncuHF6/vnnO3Xe3r17VVxcHHwlJyd39qt7jJYZNRsPnDC3EAAA+gB7Z0+YOXOmZs6c2ekvSk5OVlxcXKfP64luHN5fNqtFW4+U6Yvjlbqif4zZJQEA0GuFbMzI+PHjlZqaqltuuUV/+9vf2j22rq5Ofr+/1asnSY+P1viMOEnSn3eyEisAAJej28NIamqqXnzxRb399tt6++23lZGRoRtvvFHbt29v85ycnBy53e7gKyMjo7vL7LRhyc29Ib/4cL8CrDcCAMAlsxiGccm/pBaLRcuWLdPs2bM7dd4NN9ygzMxM/e53v7vg/rq6OtXV1QXf+/1+ZWRkyOfzyeVyXWq5XWrjgRP65m82SZLWfO8GbtUAAHAOv98vt9t90d9vU6b2Tp48WQcOHGhzv8PhkMvlavXqaaYOTdLEgfGSpF1HfSZXAwBA72VKGMnLy1NqaqoZX92lhntiJUkHSitNrgQAgN6r07NpKisrW/VqHDp0SHl5eUpISFBmZqYWLVqkoqIivfbaa5KkX/ziFxo8eLBGjRql2tpa/eY3v9HatWv1wQcfdN1VmGTo6Vsz+0srTK4EAIDeq9NhZOvWrbrpppuC7x955BFJ0ty5c7V48WIVFxeroKAguL++vl7f+973VFRUpOjoaI0dO1Yffvhhq8/orYalNIcRekYAALh0lzWANVQ6OgAm1Ly+Wk3JWSOb1aI9T8+Qw24zuyQAAHqMHj2Ata9IcTmUFONQU8DQm1sKzS4HAIBeiTByGSwWi6YNTZQkLd542NxiAADopQgjl+n704dLkg4er1Kpv9bkagAA6H0II5cpIyFao9Ka74N9cuiUydUAAND7EEa6wJQhzbdqcr/gKb4AAHQWYaQLXDcsSZK0Yd8J9YLJSQAA9CiEkS4wZXCiIu1WFZXXaMN+ekcAAOgMwkgXiIq06fph/SVJL2/4wuRqAADoXQgjXeRfbh4qSfrk4CmVMKsGAIAOI4x0kdED3BqaHKOmgKGX1h80uxwAAHoNwkgXsVktevTWEZKk13IPs+YIAAAdRBjpQtlXJWtQYrQaA4Z2HvWZXQ4AAL0CYaQLWSwWXZ0ZL0l679NjJlcDAEDvQBjpYrePS5MkrdzlVW1Dk8nVAADQ8xFGutiNw/urf6xD9U0BbTnM8vAAAFwMYaSLWSwW3TIyRZL0pzxu1QAAcDGEkW5w8/BkSdKuIgaxAgBwMYSRbjDy9FN895dW6lRVvcnVAADQsxFGukFaXJTGDHCrKWDo6n9brWJfjdklAQDQYxFGusn91w0K/vmZ9z4zrxAAAHo4wkg3uWPcAE0elCBJem9XsQIBw+SKAADomQgj3cRqteiNb09RVIRNhiH9au1+s0sCAKBHIox0I5vVIrvNIkn6xYf7VXCy2uSKAADoeQgj3exfbh4a/PM/vLhRhsHtGgAAzkYY6Wb3Txus6EibJKm0ok7bjpSZXBEAAD0LYaSb2W1WvfiPE4Pv/+l32+gdAQDgLISRELj+yv66b9ogSdLJqnr9as0BcwsCAKAHIYyEyJOzRgb//F8f7lN9Y8DEagAA6DkIIyFitVr01rys4Pt7X91sYjUAAPQchJEQumZQgjwupyRp4xcn9VruYXMLAgCgByCMhNjrD04O/vmpd3frpytZKh4AEN4IIyE2NDlWEwfGB9+/vOGg6hqbTKwIAABzEUZM8NK3JirWaQ++P8LKrACAMEYYMUFSjEO7fjJDowe4JEnT/2uDDp+oMrkqAADMQRgx0Zwpg4J//vpLuWriyb4AgDBEGDHR16/JCPaOlFbU6YrHV2r3MZ/JVQEAEFqEEZP9cd5UDU2OCb6f9/o21TYwoBUAED4IIyZzRti0fP40xTiaB7QWnqrRd17fZnJVAACEDmGkB4hx2JX/f2bosZkjJEkf7T2uQY+9p6NlzLIBAPR9hJEeZN4NVyj7quTg++v+/SOe8AsA6PMIIz3M/JuGtnr/bt4xkyoBACA0CCM9zITMeOV8dUzw/cI38/TUu/kKMO0XANBHEUZ6oLsnZ+rb1w8Jvn8t94g+2FNiYkUAAHQfwkgP9eitIzQna2Dw/Tvbj7IoGgCgTyKM9FA2q0U/uX2UXrl3kiwW6YM9JRr51Cq9s/2oNh08aXZ5AAB0GcJID2a1WnTziBTdO3WQJKmuMaBH/rBTd738id7NKzK3OAAAughhpBd4ctZIjRngbrXt4aV5mvc7FkcDAPR+hJFewGq16K15Wa3GkEjSqt1e5RfxLBsAQO9GGOklnBE2PX3HaH36k+mttv/9rz/WtiNlJlUFAMDlI4z0Mi5nhD57+tZW21buKjapGgAALl+nw8iGDRt0++23Ky0tTRaLRcuXL7/oOevWrdPVV18th8OhoUOHavHixZdQKlpERdr01rys4PvffnxI33l9mz76vFT/tXqfqusbTawOAIDO6XQYqaqq0rhx4/T888936PhDhw5p1qxZuummm5SXl6eFCxfqwQcf1Pvvv9/pYnHGNYMSdPCnt+nWUR5J0l/yvbpv8Rb9cs1+jXzqfWX/fL2KymtMrhIAgIuzGJfxJDaLxaJly5Zp9uzZbR7z6KOP6r333lN+fn5w2ze+8Q2Vl5dr1apVHfoev98vt9stn88nl8t1qeX2SYZh6P3dJZr3+oVn1nx/+pVacPOwEFcFAEDHf7+7fcxIbm6usrOzW22bMWOGcnNz2zynrq5Ofr+/1QsXZrFYdOtoj96al6V5N1xx3v7//GCfHl+2S4Wnqk2oDgCAi+v2MOL1epWSktJqW0pKivx+v2pqLnwbIScnR263O/jKyMjo7jJ7vWsGJeixmSP0wXev119/eJMevG5wcN+STQW688Vc5RWWy+urNbFKAADO1yNn0yxatEg+ny/4KiwsNLukXuPKlFhlJETr8duu0v/58qjgdq+/VrOf/5um5KzRqnyvLuPuHAAAXarbw4jH41FJSesnzpaUlMjlcikqKuqC5zgcDrlcrlYvdI7VatHcqYP0xU9v0/enX9lq37zXt2nwopX6885jJlUHAMAZ3R5GsrKytGbNmlbbVq9eraysrDbOQFeyWS164LohGpgYfd6+f3ljh37x4T6t21uqvd4KE6oDAECyd/aEyspKHThwIPj+0KFDysvLU0JCgjIzM7Vo0SIVFRXptddekyTNmzdP//3f/60f/vCHuv/++7V27Vr94Q9/0Hvvvdd1V4F2RUXa9Od/uU6+6gbNfXWzDh6vCu77xYf7g39etfDvNMLjUuGpakVH2pQY4zCjXABAmOn01N5169bppptuOm/73LlztXjxYt177706fPiw1q1b1+qc7373u9qzZ4/S09P15JNP6t577+3wdzK1t+vt9Vbo27/bqiMnW8+yGeGJ1eene0l2/ni63FERZpQHAOgDOvr7fVnrjIQKYaR71DU26Xe5R/TB7hJtPnzqvP1x0RH66w9vUrGvVu/mFWn+TUMVHdnpzjQAQJgijKBT/t/Gw3p6xR41Bdr+6zD/pis0c3SqhibHyBlhC2F1AIDeiDCCTqttaJJhSG9vP6ofLc9v99hl/zxVEzLjJTWvAtsUMGS39ciZ4gAAk3T095s+dwS19Hbcc22mPvf69fonBW0e+5X/2SiPy6nbxqTqZFWdNuw7rlULr1eKyxmqcgEAfQQ9I+iQsqp6/XLNfi3eeLjNY4b076el356i5FgCCQCA2zToBoGAoT9sLdTijYeV7HJqw77jFzzuiduu0uCkfnp90xEdLavRGw9NUf9YpgkDQLghjCAkBj3WsfVifjTrKn3z2kxm4wBAGCGMICR81Q266+Vcfe6t0Nh0t6YMSdTLGw62eXysw667r83U96cPV6TdqqaAIaul+enDAIC+hTCCkKpvDEiSIu1WVdQ26O1tR/WTP+9p95xIu1X1jQHdO3WQvnRVsq4ZlMCUYQDoQwgj6DF+uvKzdntLznb35EztKCjTj2aNlCvKrjED3PSaAEAvRRhBjxEIGDpRVafkWKd2Fpbrza2FSoiO1H9/dODiJ0t6+VsTNT4zjlk6ANDLEEbQ4/mqG/RfH+7TriKf9pdUyF/b2O7xt4xMUVJMpL4+KUPj0uNktdJjAgA9GWEEvYphGPreWzv1zvaiDh3vsFv1+G1Xqay6Xn/eeUxfvTpd35ycqfh+kd1cKQCgowgj6JW2F5SpX6Rdwz2x2njghIrKa/TyhoPaX1rZofMnDYzXM18ZoxOVdXpi2S7NGO3RD6YPl9VioScFAEKMMII+ZUdBmTISorUq33vR5+a05T++Nlb9Yx1avPGw7rk2U9NHebq4SgDA2Qgj6LPKq+u1cpdXAxOjtWRzgVJinTp4olLr9l54Rdi2PPvVMRo9wK0BcVHaXlCmd/OO6SdfHqV+DptK/XXKSIjupisAgPBAGEFY2nzolEoratUUMPSrNfslSV8cr+rw+V+dMED+2kZ9+FmJfjBjuL45OVM7Css0eXCiYhysHgsAnUEYAU47fKJKq3Z7tavIp7/uO37RWTsXkpkQrX+YmK5hyTFqDBganxFHzwkAXARhBGhDU8DQf36wV5sOntSoNLf+duCE/n5sqsqqG/S7T450+HMemzlCdQ0BNTQFdOtoj46V12jt56V6OHuYUt1RKvHXKtZp53k8AMIWYQToJMMw9OFnpZKkCJtFf91/QjEOu/647aiKymsu+XP/6foh8tc2qKa+ST++fVRw+nFDU0B2q4UVZgH0WYQRoIuU+mv19vYivbDugDITo/UPV6cr/5hff9x2tEs+/wczhispJlIffX5cT8y6Sikup0orapUez20gAL0bYQToYoGAIUOS7fR6JYZhaH9ppTbsO67/+95nkqQ7J6arpKJOG/Z1bmZPWyLtVj1yy5XaerhMh05UKntkinYUlGvL4VP6/YPXauoVSV3yPQDQHQgjQIg1BYxgUGl5X13fqNqGgHL+8pn+vPOYYp0RGuGJ1Zh0t2rqm/RabsfHqFxIqtup4xV1evqO0SqrrlfhqWotzL5SHjfP8QFgPsII0AtU1Dao2FerLYdP6Ylll7aY24XMGJWi1XtKFDCk7KuS9R//ME4Ou1Wv/u2Qbh3t0dDk2FbH7yup0MDEaDnsNtU2NMkZYeuyWgCEL8II0IsVnKxWdUOjUmKd2l5Qpsq6Rk0bmqRbfr5ekXarXv7WJP3jbzep4hKmKUvS3w1L0qdHfaptaFJdY0CS1C/SpnumDNRv/npQY9Lj9N3sYfLVNCjGYdeXrkqRJB0rr1FiTKQibVYG3gK4KMII0AdV1zeqMWDI5YxQdX2jDENyRtj09vajamwy5PXXBhd760pThiTo/mmD9e3fbQtum3pFor59/RCt33dcf8o7ptQ4p5796liNSnMRVABIIowAYauxKaCDJ6qUmRCt7QVl+ssur0oravX+7hJJ0lN/P1L7Syv0xubCbqthWHKMyk/3qqS6naqsa5Qzwqb9JRX6yoR0zZ6QpqHJMazBAvRxhBEA7aptaJLFIjnsNgUChkor6vThZyW6MiVWK3cVa/HGwyGr5Z+uH6JvTM5Uv0ibtheUacqQRMVFRwb3F56q1hfHK3Xj8OTgNsMw1Bgw1BQwGOMC9FCEEQCXpbahSXarRTarRQ+9tk0fflYim9Wiof1jtLekQt+8NlP3TxusVLdTh09W6fFl+dpZWN5l3x8XHaHy6gbFOu3BsTEup11j0+P0WbFfVadnKknS0OQYfWXCAN09OVMup13r9x3X6AFuxTrtioqwyV/TKJvNorKqepbxB0KIMAKgywQCzb0QkXZru8c1BQzVNTYpOtKuhqaADp2o0uZDp+RxOXVFcoyOV9Tp31d9rm1HypSZEK3hnlit3lMSoqtoNiw5RjPHpMoi6fork7Sz0Kei8hpNGhiv0oo6fXlcWnCV3BaNTQFV1TXJHR0R0lqB3o4wAqDXqKxr1Iqdx/TYO7skSbeN8WjlLq9GeGJ1ZUqsNn5xQicq60NWzw9mDNcnB08q94uTagyc+Sdy0sB4DUuJ0fRRHt14ZX9ZLBadqKxTfpFPU69IumhYA8INYQRAr3P4RJX6OezqH+s4b99L67/QSxsO6pV7r1F+kU9XpsTKX9OgiQPjVXCqWier6rTlcJkOHa/S1QPjVNcQ0C/X7Ff2VSnaV1KhgyeqQnYdYwa4VdfYpH0llRrhidVTfz9SH+wp0d8OnFBUpE0zRnlks1rUFDD0nRuukNVqkWEYrWYhlVXVa29JhdxREboqlX/30DsRRgDgNMMwtL2gXHHREfpd7hFNGhSv6Eib3thcqJr6JiW7HJqQEadIu1XvbC9SXmG5HHar/Je4jktnDUyM1pGT1ZKkqAibahqaLnjcq/ddo6sz47W7yKfjlXX647ajckbYNDAhWvdfN1hpcVEXPK+ovEZRETYlnHP7CehuhBEAuETn9lIYhqG9JRU6Vl4jX02DImxW7Trq00sbDgaPcUdFyGa16FRV6G4ntSU51qGy6nplJkTrxuHJ+u3Hh5TmduqfbrhCIzyxemNzgWobAnro+iEam+6W3WrRyl1eDUqK1qg0t46WVcsdFaFY5/ljZMqq6hUXHcFaMugQwggAhMDnXr8GJ/WTw948vbjYV6OdheVauqVQg5P6aV9JhXYW+nTt4AT9/Ovj9dcDx7VgyQ5JzQvHfW/6lTpeUa9TVfX60fJdCpzzL/JVqS4NSozWX/K9ob40RdqsumZwvG4d5dGmQ6e04tNiSVJ6fJRmjx+gLYdPaefRck29Ikk3j0jW7ePSFAgYChiGin21Gu6JVX1jQP0crCcTrggjANCDHTlZpeRYp6Iiz18jZfcxnzwupxJjzoyd2XTwpJ5d9blOVNbpH67OkMft0KNv79LQ5BgdKK087zNapkb3FE/cdpV+vXZ/8NbXkKR++ua1mfL6arVh/3HtK6lUZkK07p82SPH9IjUqzSVXVISSY5sf+phXWK60OKdiHRGy2yx67v29mjIkQdcP6y9DUoSNwcM9EWEEAMJEZV2jKmsb5XE7FQg0PxagZfyI11erhH6R+mhvqXw1DfLXNGhI/34qPFWjH/9ptyTpX28eqoq6Rv3twAntKzk/2PRGIzyxmj7KI4fdqrKqes2eMEAjU12yWi3afOiUJOmaQfGtbjf5qhskS/Mtt6aAIatF592u4/ZU5xBGAADtqmtsks1ikf2cXoWCk9UqKq9R1hWJMgxD7+YdU7LLoalXJMkwDPlrGvX9P+5UhM2in9w+SnabVW9uKZTDbtWhE1XaU+zXtiNlSuwXqZMXGEOTHh8lZ4RNBaeqVX/6QY1mibRbz6uhX6RNVfVNGpzUTwtuGqojJ6uUHh+tX3+0Xx6XU9cMSlBcdIRuG5Mqu9WqX67Zp0+P+jRrbKpK/XX62tXp8ridSoppHjBssVhU3xgIy6nfhBEAgGl81Q2y2Sxy2q2qbQyoqKxGybGO8xaU81U3qK6pSV5frdLjo7X7mE9/3HZUY9PjtPHACa35vDR47NQrErXxi5OhvpQuk5EQpZuHJ+toWY1OVNZp51GfJOnOiemqrGtU/1iHKmsbta+0QmMGxKnUXyu7zaLJgxNV3xhQenyUUt1ODU2OUT+HXXu9FRqU1E/f/8NOJcRE6ltTBurTo+X6yoR0RdgsqqhrlEVSrDNC+UU+DUyMvuCg5LzCciX2i+yW1YkJIwCAXu9EZZ1iHPbg84eq6hpls1pU3xTQH7YUamSaSxMy4hUwDH1y8GQw7GTERwcX0/O4naqobdTTK/a0+uxRaS7ZrBZ9ejoUnM0ZYdXYAXEqKq9RUXlN919oCNisFmXER6m6vkkBo7ltz/Y/91yt28akdul3EkYAADhLbUOTImxW2awXHvfhq2lQY1Og1cDhFvtKKjQgLioYiqwW6VRVvfZ6KzQgPkqZp3sVjpbVaMGS7cFeD+nMbZ+zOSOswWcr9RTbn7yly9eiIYwAAGCyxqaA7DarSitq5a9p1NDkGPlqGhTrsKuqvlERNqucEbbg4NhtR8q0fEeRRqa5dNvoVB2vrNP/23hYHrdTeYXlWr2nRFlDElXir9WkQfE6XlGnhH4ORdqtemNzQbu1eFxOef21F9z3LzcP1femD+/y6yeMAADQh9Q3BlTb2CTXBcZ9nKu2oUkWi1RT36Rj5bUamdb6t/Pj/Se0v7RCo9Lcmjw4obtK7vDvNyvRAADQC0TarR2ekdNyO8lhtyku+vxbL9cNS9J1w5K6tL7LEX7zjAAAQI9CGAEAAKYijAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMNUlhZHnn39egwYNktPp1LXXXqvNmze3eezixYtlsVhavZxO5yUXDAAA+pZOh5E333xTjzzyiH784x9r+/btGjdunGbMmKHS0tI2z3G5XCouLg6+jhw5cllFAwCAvqPTYeTnP/+5HnroId13330aOXKkXnzxRUVHR+uVV15p8xyLxSKPxxN8paSkXFbRAACg7+hUGKmvr9e2bduUnZ195gOsVmVnZys3N7fN8yorKzVw4EBlZGTojjvu0O7du9v9nrq6Ovn9/lYvAADQN3UqjJw4cUJNTU3n9WykpKTI6/Ve8Jzhw4frlVde0bvvvqvXX39dgUBAU6dO1dGjR9v8npycHLnd7uArIyOjM2UCAIBepNtn02RlZWnOnDkaP368brjhBr3zzjvq37+/XnrppTbPWbRokXw+X/BVWFjY3WUCAACTdOqpvUlJSbLZbCopKWm1vaSkRB6Pp0OfERERoQkTJujAgQNtHuNwOORwOILvDcOQJG7XAADQi7T8brf8jrelU2EkMjJSEydO1Jo1azR79mxJUiAQ0Jo1a7RgwYIOfUZTU5N27dql2267rcPfW1FRIUncrgEAoBeqqKiQ2+1uc3+nwogkPfLII5o7d64mTZqkyZMn6xe/+IWqqqp03333SZLmzJmjAQMGKCcnR5L09NNPa8qUKRo6dKjKy8v13HPP6ciRI3rwwQc7/J1paWkqLCxUbGysLBZLZ0tuk9/vV0ZGhgoLC+Vyubrsc/si2qpjaKeOo606hnbqONqqY0LZToZhqKKiQmlpae0e1+kwctddd+n48eN66qmn5PV6NX78eK1atSo4qLWgoEBW65mhKGVlZXrooYfk9XoVHx+viRMnauPGjRo5cmSHv9NqtSo9Pb2zpXaYy+XiL24H0VYdQzt1HG3VMbRTx9FWHROqdmqvR6SFxbjYjZw+zO/3y+12y+fz8Rf3ImirjqGdOo626hjaqeNoq47pie3Es2kAAICpwjqMOBwO/fjHP241cwcXRlt1DO3UcbRVx9BOHUdbdUxPbKewvk0DAADMF9Y9IwAAwHyEEQAAYCrCCAAAMBVhBAAAmCqsw8jzzz+vQYMGyel06tprr9XmzZvNLimkcnJydM011yg2NlbJycmaPXu29u7d2+qY2tpazZ8/X4mJiYqJidHXvva1855NVFBQoFmzZik6OlrJycn6wQ9+oMbGxlBeSkg9++yzslgsWrhwYXAb7dSsqKhI//iP/6jExERFRUVpzJgx2rp1a3C/YRh66qmnlJqaqqioKGVnZ2v//v2tPuPUqVO655575HK5FBcXpwceeECVlZWhvpRu1dTUpCeffFKDBw9WVFSUrrjiCv3bv/1bq+d3hGtbbdiwQbfffrvS0tJksVi0fPnyVvu7ql0+/fRT/d3f/Z2cTqcyMjL0H//xH919aV2qvXZqaGjQo48+qjFjxqhfv35KS0vTnDlzdOzYsVaf0aPayQhTS5cuNSIjI41XXnnF2L17t/HQQw8ZcXFxRklJidmlhcyMGTOMV1991cjPzzfy8vKM2267zcjMzDQqKyuDx8ybN8/IyMgw1qxZY2zdutWYMmWKMXXq1OD+xsZGY/To0UZ2draxY8cOY+XKlUZSUpKxaNEiMy6p223evNkYNGiQMXbsWOPhhx8ObqedDOPUqVPGwIEDjXvvvdfYtGmTcfDgQeP99983Dhw4EDzm2WefNdxut7F8+XJj586dxpe//GVj8ODBRk1NTfCYW2+91Rg3bpzxySefGH/961+NoUOHGnfffbcZl9RtnnnmGSMxMdFYsWKFcejQIeOtt94yYmJijF/+8pfBY8K1rVauXGk88cQTxjvvvGNIMpYtW9Zqf1e0i8/nM1JSUox77rnHyM/PN9544w0jKirKeOmll0J1mZetvXYqLy83srOzjTfffNP4/PPPjdzcXGPy5MnGxIkTW31GT2qnsA0jkydPNubPnx9839TUZKSlpRk5OTkmVmWu0tJSQ5Kxfv16wzCa/0JHREQYb731VvCYzz77zJBk5ObmGobR/D+E1Wo1vF5v8JgXXnjBcLlcRl1dXWgvoJtVVFQYw4YNM1avXm3ccMMNwTBCOzV79NFHjeuuu67N/YFAwPB4PMZzzz0X3FZeXm44HA7jjTfeMAzDMPbs2WNIMrZs2RI85i9/+YthsViMoqKi7is+xGbNmmXcf//9rbZ99atfNe655x7DMGirFuf+yHZVu/zP//yPER8f3+r/vUcffdQYPnx4N19R97hQaDvX5s2bDUnGkSNHDMPoee0Ulrdp6uvrtW3bNmVnZwe3Wa1WZWdnKzc318TKzOXz+SRJCQkJkqRt27apoaGhVTuNGDFCmZmZwXbKzc3VmDFjgs8mkqQZM2bI7/dr9+7dIay++82fP1+zZs1q1R4S7dTiT3/6kyZNmqQ777xTycnJmjBhgv73f/83uP/QoUPyer2t2sntduvaa69t1U5xcXGaNGlS8Jjs7GxZrVZt2rQpdBfTzaZOnao1a9Zo3759kqSdO3fq448/1syZMyXRVm3pqnbJzc3V9ddfr8jIyOAxM2bM0N69e1VWVhaiqwktn88ni8WiuLg4ST2vnTr9oLy+4MSJE2pqamr1wyBJKSkp+vzzz02qylyBQEALFy7UtGnTNHr0aEmS1+tVZGRk8C9vi5SUFHm93uAxF2rHln19xdKlS7V9+3Zt2bLlvH20U7ODBw/qhRde0COPPKLHH39cW7Zs0b/+678qMjJSc+fODV7nhdrh7HZKTk5utd9utyshIaHPtJMkPfbYY/L7/RoxYoRsNpuampr0zDPP6J577pEk2qoNXdUuXq9XgwcPPu8zWvbFx8d3S/1mqa2t1aOPPqq77747+CyantZOYRlGcL758+crPz9fH3/8sdml9DiFhYV6+OGHtXr1ajmdTrPL6bECgYAmTZqkn/70p5KkCRMmKD8/Xy+++KLmzp1rcnU9yx/+8Af9/ve/15IlSzRq1Cjl5eVp4cKFSktLo63QpRoaGvT1r39dhmHohRdeMLucNoXlbZqkpCTZbLbzZjuUlJTI4/GYVJV5FixYoBUrVuijjz5Senp6cLvH41F9fb3Ky8tbHX92O3k8ngu2Y8u+vmDbtm0qLS3V1VdfLbvdLrvdrvXr1+tXv/qV7Ha7UlJSaCdJqampGjlyZKttV111lQoKCiSduc72/r/zeDwqLS1ttb+xsVGnTp3qM+0kST/4wQ/02GOP6Rvf+IbGjBmjb33rW/rud7+rnJwcSbRVW7qqXcLh/0fpTBA5cuSIVq9e3eoJvT2tncIyjERGRmrixIlas2ZNcFsgENCaNWuUlZVlYmWhZRiGFixYoGXLlmnt2rXndcdNnDhRERERrdpp7969KigoCLZTVlaWdu3a1eovdctf+nN/mHqrL33pS9q1a5fy8vKCr0mTJumee+4J/pl2kqZNm3be1PB9+/Zp4MCBkqTBgwfL4/G0aie/369Nmza1aqfy8nJt27YteMzatWsVCAR07bXXhuAqQqO6ulpWa+t/fm02mwKBgCTaqi1d1S5ZWVnasGGDGhoagsesXr1aw4cP7zO3aFqCyP79+/Xhhx8qMTGx1f4e105dPiS2l1i6dKnhcDiMxYsXG3v27DG+/e1vG3Fxca1mO/R13/nOdwy3222sW7fOKC4uDr6qq6uDx8ybN8/IzMw01q5da2zdutXIysoysrKygvtbpqxOnz7dyMvLM1atWmX079+/T01ZvZCzZ9MYBu1kGM2j9e12u/HMM88Y+/fvN37/+98b0dHRxuuvvx485tlnnzXi4uKMd9991/j000+NO+6444LTMidMmGBs2rTJ+Pjjj41hw4b1+umq55o7d64xYMCA4NTed955x0hKSjJ++MMfBo8J17aqqKgwduzYYezYscOQZPz85z83duzYEZwF0hXtUl5ebqSkpBjf+ta3jPz8fGPp0qVGdHR0r5ra21471dfXG1/+8peN9PR0Iy8vr9W/72fPjOlJ7RS2YcQwDOPXv/61kZmZaURGRhqTJ082PvnkE7NLCilJF3y9+uqrwWNqamqMf/7nfzbi4+ON6Oho4ytf+YpRXFzc6nMOHz5szJw504iKijKSkpKM733ve0ZDQ0OIrya0zg0jtFOzP//5z8bo0aMNh8NhjBgxwnj55Zdb7Q8EAsaTTz5ppKSkGA6Hw/jSl75k7N27t9UxJ0+eNO6++24jJibGcLlcxn333WdUVFSE8jK6nd/vNx5++GEjMzPTcDqdxpAhQ4wnnnii1Q9FuLbVRx99dMF/l+bOnWsYRte1y86dO43rrrvOcDgcxoABA4xnn302VJfYJdprp0OHDrX57/tHH30U/Iye1E4WwzhryT8AAIAQC8sxIwAAoOcgjAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqQgjAADAVP8f4Aty07LcoBkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor(l).view(-1, 10).mean(1).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cPqZDUJ8I2AA",
    "outputId": "a6adc02a-a989-4bc5-ab8b-8840be56238d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def _initialModel._to_java_impl():\n",
      "        \"\"\"\n",
      "        Deprecated in 2.3.0. Use :func:`pyspark.sql.types.DataType`, int or :class:`Column` expression in the given key (default param).\n",
      "\n",
      "        >>> df = spark.range(1, 0).alias('age')).collect()\n",
      "        [Row(name=u'Alice', age=1, name=u'Alice')]\n",
      "        \"\n"
     ]
    }
   ],
   "source": [
    "# Text generation\n",
    "begin_text = torch.tensor(tok.encode('def '), device=device).unsqueeze(0)\n",
    "print(''.join(tok.decode(generate_batch(model, begin_text))))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "V100",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
